Free or Paid, Rates of Google Play Store Users¶

# Read dataframe and display data

df = pd.read_csv('googleplaystore.csv')

df.head(5)

df.shape

df.columns

df.describe

df.boxplot()

df.hist()

df.info()

df.isnull()

df.isnull().sum()

df[df.Rating>5]

df.drop([10472],inplace=True)

df[10470:10475]

df.boxplot()

df.hist()

​

# check duplicates

n_duplicated = df.duplicated(subset=['App']).sum()

print("There are {}/{} duplicated records.".format(n_duplicated, df.shape[0]))

df_no_dup = df.drop(df.index[df.App.duplicated()], axis=0)

print("{} records after dropping duplicated.".format(df_no_dup.shape[0]))

# Check and clean type values, defer nan value processing to the next cell

print(set(df_no_dup.Type))

print("Dropping alien Type value '0', {} record(s) removed".format(sum(df_no_dup.Type == '0')))

df_no_dup = df_no_dup.drop(df_no_dup.index[df_no_dup.Type == '0'], axis=0)

# check and drop NaN values

print("NaA value statistics in each column")

print(df_no_dup.isnull().sum(axis=0),'\n')

df_no_dup = df_no_dup.dropna(subset=['Type'])

print("Column 'Type' with NaN values are dropped, {} records left.".format(df_no_dup.shape[0]))

​

# prepare rating dataframe

df_rating = df_no_dup.dropna(subset=['Rating'])

print("Cleaned dataframe for 'Rating' has {} records.".format(df_rating.shape[0]))

# we are interested in the columns Category, Rating and Type

# Drop irrelevant columns for Rating dataframe.

df_rating = df_rating.loc[:,['Rating', 'Type', 'Category']]

def plot_hist(df, col, bins=10):

    """

    Plot histograms for a column

    """

    plt.hist(df[col], bins=bins)

    plt.xlabel(col)

    plt.ylabel('counts')

    plt.title('Distribution of {}'.format(col))

​

def compute_app_types(df):

    """

    Given a dataframe, compute the number 

    of free and paid apps respectively

    """

    return sum(df.Type == "Free"), sum(df.Type == 'Paid')

​

def plot_app_types(df):

    """

    Plot app type distributions across categories

    """

    vc_rating = df.Category.value_counts()

    cat_free_apps = []

    cat_paid_apps = []

    for cat in vc_rating.index:

        n_free, n_paid = compute_app_types(df.query("Category == '{}'".format(cat)))

        cat_free_apps.append(n_free)

        cat_paid_apps.append(n_paid)

​

    f, ax = plt.subplots(2,1)

    ax[0].bar(range(1, len(cat_free_apps)+1), cat_free_apps)

    ax[1].bar(range(1, len(cat_free_apps)+1), cat_paid_apps)

​

def drop_categories(df):

    """

    Drop categories with any app type with instances fewer than 10

    """

    vc_rating = df.Category.value_counts()

    cats_to_drop = []

    for cat in vc_rating.index:

        n_free, n_paid = compute_app_types(df.query("Category == '{}'".format(cat)))

        if n_free < 10 or n_paid < 10:

            cats_to_drop.append(cat)

    for cat in cats_to_drop:

        df.drop(df.query('Category == "{}"'.format(cat)).index, axis=0, inplace=True)

    print("Deleted categories: {}".format(cats_to_drop))

    return df

# Describe Rating dataframe

plot_hist(df_rating, 'Rating')

df_rating.describe()

print("There are {} free and {} paid apps in the the Rating dataframe ".format(*compute_app_types(df_rating)))

# explore the distributions of free and paid apps across different categories

plot_app_types(df_rating)

# Exclude categories with fewer than 10 apps for any Free or Paid type

# Otherwise the categories would contain too few data to generalize the result

df_rating = drop_categories(df_rating)

print("Cleaned Rating dataframe has {} datapoints".format(df_rating.shape[0]))

df_rating.describe()

def plot_target_by_group(df, target_col, group_col, figsize=(6,4), title=""):

    """

    Plot the mean of a target column (Numeric) groupped by the group column (categorical)

    """

    order = sorted(list(set(df[group_col])))

    stats = df.groupby(group_col).mean()[target_col]

    fig, ax = plt.subplots(figsize=figsize)

    sns.barplot(x=group_col, y=target_col, data=df, ax=ax, order=order).set_title(title)

    ax.set(ylim=(3.8, 4.5))    

    return stats

stats = plot_target_by_group(df_rating, 'Rating', 'Type', title="Average Rating Groupped by App Type")

for i, s in zip(stats.index, stats):

    print("{} app has average {} {}".format(i, 'Rating',s))

mean_rating = df_rating.Rating.mean()

print("Mean rating: {}".format(mean_rating))

paid_stats = plot_target_by_group(df_rating.query('Type == "Paid"'), 'Rating', 'Category', (16, 4), "(Paid App) Average Ratings by App Category")

free_stats = plot_target_by_group(df_rating.query('Type == "Free"'), 'Rating', 'Category', (16, 4), "(Free App) Average Ratings by App Category")

fig, ax = plt.subplots(figsize=(16,4))

sorted_idx = sorted(paid_stats.index)

rating_diff = paid_stats[sorted_idx] - free_stats[sorted_idx]

sns.barplot(x=sorted_idx, y=rating_diff, ax=ax).set_title("Difference of Ratings between Paid and Free Apps Across App Categories");

rating_diff

def compute_utest(df):

    """

    Compute Mann-Whitney rank tests

    for paid and free app ratings

    """

    paid_rating = df.query('Type == "Paid"')['Rating']

    free_rating = df.query('Type == "Free"')['Rating']

    return mannwhitneyu(paid_rating, free_rating)

​

def cat_utest(df):

    """

    Iteratively compute utest for each app category

    """

    cats = set(df.Category)

    res = []

    for cat in cats:

        stats, pval = compute_utest(df.query('Category == "{}"'.format(cat)))

        res.append({'Category':cat,

                    'u_statistics':stats,

                    'p_value':pval})

    return pd.DataFrame(res)

​

uval, pval = compute_utest(df_rating)

​

print("General utest result: pval {}, u {}".format(pval, uval))

df_utest = cat_utest(df_rating)   

df_utest.loc[df_utest.p_value < .05] # significant categories

# Visualization

import matplotlib.pyplot as plt

import plotly

import plotly.express as px

import plotly.graph_objects as go

import seaborn as sns

​

# Model

from sklearn.cluster import KMeans

from sklearn.cluster import DBSCAN

name = df['Category'].value_counts().index

num = df['Category'].value_counts().values

​

fig = px.pie(data_frame=df,names=name,values=num,

             title='Pies chart show application categories.',width=1200,height=600)

fig.update_traces(textposition='inside',textinfo='label+percent')

fig.show()

plt.figure(figsize=(18,6))

sns.barplot(x='Category',y='Rating',data=df)

plt.xticks(rotation=90)

plt.title('Application category Vs Average Rating',fontsize=15)

plt.xlabel('Application Catagory',fontsize=12)

plt.ylabel('Rating',fontsize=12)

plt.ylim(0,5)

plt.show()

plt.figure(figsize=(18,6))

sns.scatterplot(x='Category',y='Rating',data=df,s=40)

plt.xticks(rotation=90)

plt.title('Application category Vs Rating (Scatter)',fontsize=15)

plt.xlabel('Application Catagory',fontsize=12)

plt.ylabel('Rating',fontsize=12)

plt.ylim(0.5,5.5)

plt.show()

x = df['Type'].value_counts().index

y = df['Type'].value_counts().values

​

fig = px.pie(data_frame=df,names=x,values=y,width=800,height=600

             ,title='Types of application.')

fig.update_traces(textposition='inside',textinfo='label+percent')

fig.show()

#dropna

df.dropna(inplace=True)

data = df.copy()

data.head()

from collections import defaultdict

from sklearn.preprocessing import LabelEncoder

#LabelEncoder

d = defaultdict(LabelEncoder)

data = data.apply(lambda x : d[x.name].fit_transform(x))

data.head()

# K-Means

X = data[['App','Category','Rating','Reviews','Size','Installs','Type','Price','Content Rating','Genres']]

model = KMeans(n_clusters=5).fit(X)

y_kmeans = model.fit_predict(X)

df['Predicted_Cluster'] = y_kmeans

​

# Visualizing all the cluster

plt.figure(figsize=(15,8))

sns.scatterplot(x='Category',y='Rating',hue='Predicted_Cluster',data=df,s=50)

plt.xticks(rotation=90)

plt.show()

​